---
title: "Benchmarking_DataGeneration"
author: ""
date: ""
output: html_document
---

```{r setup, include=FALSE}

rm(list = ls())

library(tidyverse)
library(httr)
library(jsonlite)
library(openxlsx)
library(pbapply)
library(tictoc)

```

# Undergrad values and benchmarks

```{r load and clean}
#Load in data for each coder 
cd_cod = read_csv("Undergrad_codings_coder2 - Sheet1.csv") 

aw_cod = read_csv("Undergrad_codings_coder3 - Sheet1.csv") 

ar_cod = read_csv("Undergrad_codings_coder1 - Sheet1.csv") 

group_cod = read_csv("Group_ROCCA_codings - Sheet1.csv") 

tt = rbind(aw_cod, ar_cod, cd_cod, group_cod)

#Create 4 label models
tt = tt %>% mutate(
  human = as.integer(MAIN_HUM_HUMAN == 1 | MAIN_HUM_REFUGE == 1),
  threat = as.integer(MAIN_THREAT_DIS == 1 | MAIN_THREAT_ECON == 1 | 
                      MAIN_THREAT_VIOL == 1 | MAIN_THREAT_INSTAB == 1), 
  benefit = as.integer(MAIN_BEN_ECON == 1), 
  pi = as.integer(MAIN_PI == 1)) %>% 
  mutate(across(c(MAIN_HUM_HUMAN, MAIN_HUM_REFUGE, MAIN_THREAT_DIS, MAIN_THREAT_ECON,
                  MAIN_THREAT_VIOL, MAIN_THREAT_INSTAB, MAIN_BEN_ECON, MAIN_PI), as.numeric))

#Change data format so that there are only three columns...one for each coder...and that each row corresponds with a 1/0 code for a theme
tt_coders = tt %>% 
  dplyr::select(c(human, threat, benefit, pi, CODER, DOCID, MAIN_HUM_HUMAN, MAIN_HUM_REFUGE, MAIN_THREAT_DIS,
                  MAIN_THREAT_ECON, MAIN_THREAT_VIOL, MAIN_THREAT_INSTAB, MAIN_BEN_ECON, MAIN_PI)) %>% 
  pivot_longer(cols = c(human, threat, benefit, pi, MAIN_HUM_HUMAN, MAIN_HUM_REFUGE,
                        MAIN_THREAT_DIS, MAIN_THREAT_ECON, MAIN_THREAT_VIOL, MAIN_THREAT_INSTAB, 
                        MAIN_BEN_ECON, MAIN_PI))

#Now separate out by coder
# Separate out by coder and rename columns
coders_list <- list("AW", "AR", "CD", "all") %>%
  setNames(paste0("CODER_", .)) %>%
  purrr::map(~ {
    tt_coders %>%
      filter(CODER == .x) %>%
      drop_na() %>%
      rename(!!paste0("CODER_", .x) := value) %>%
      select(-CODER)
  })

# Rejoin by DOCID and theme (name)
coders <- coders_list %>%
  purrr::reduce(full_join, by = c("DOCID", "name"))

# Clean full codings df for analysis
coders <- coders %>%
  relocate(name, .after = CODER_AW) %>%
  rename(theme = name)

rm(list = setdiff(ls(), "coders"))
```


```{r merge text with codings, eval = F}
#Note that I do not provide the rnums and all_news files in this replication code, because they contain the raw text of the newspaper articles. 

#Load in text
#text = read_csv("rnums.csv") %>% select(-c(...1))

#Load in full set of documents
#all = read_csv("all_news.csv") %>% select(-c(...1))

#Merge
#text2 = left_join(text, all, by = c("Title" = "title", "Date" = "date"), multiple = "first") %>% 
#  mutate(rnum = as.numeric(rnum))

#No mismatches
#anti_join(text, all, by = c("Title" = "title", "Date" = "date"))

#Merge full text files with codings/labels
#coders = left_join(coders, text2, by = c("DOCID" = "rnum")) %>% 
#  dplyr::select(-c(Path, text, Title, Date, url, term, month))
```


```{r create human agreement data}
# Function to determine agreement, ignoring NAs
check_agreement_ignore_na <- function(codings) {
  valid_codings <- na.omit(codings)  # Remove NA values
  if(length(unique(valid_codings)) <= 1) "Agree" else "Disagree"
}

# Function to determine agreed value and the coders who agreed
get_agreed_value <- function(codings) {
  valid_codings <- na.omit(codings)  # Remove NA values
  if(length(unique(valid_codings)) == 1) {
    return(valid_codings[1])  # Return the agreed-upon value
  } else {
    return(NA)  # Return NA in case of disagreement or no coding
  }
}

#Create agreed values
coders = coders %>%
  rowwise() %>%
  mutate(code_agreement = check_agreement_ignore_na(c(CODER_AR, CODER_AW, CODER_CD,
                                                      CODER_all)), 
         agreed_values = get_agreed_value(c(CODER_AR, CODER_AW, CODER_CD, CODER_all)))

#Create final undergraduate labels based on agreed values
coders = coders %>%
  mutate(
    label = case_when(
      code_agreement == "Agree" ~ agreed_values,
      !is.na(CODER_all) ~ CODER_all,
      !is.na(CODER_CD) ~ CODER_CD,
      !is.na(CODER_AW) ~ CODER_AW,
      !is.na(CODER_AR) ~ CODER_AR,
      TRUE ~ NA_real_
    )
  )

table(coders %>% filter(!(theme %in% c("human", "pi", "threat", "benefit"))) %>%
        dplyr::select(code_agreement)) #2.8% disagreements at label level for 8 label model

table(coders %>% filter(theme %in% c("human", "pi", "threat", "benefit")) %>%
        dplyr::select(code_agreement)) #3.9% disagreements at label level for 4 label model

#Save files
#Break into two datasets...one with 8 labels and one with 4 labels
df_four = coders %>% filter(theme %in% c("human", "pi", "threat", "benefit")) 

df_ten = coders %>% filter(!(theme %in% c("human", "pi", "threat", "benefit")))

#Save classification information into object (four label)
training = tibble(value = df_four$label, label = df_four$theme, 
                  docid = df_four$DOCID) 

training = training %>% 
  pivot_wider(names_from = label, values_from = value, values_fn = max) %>%
  arrange(docid) %>% mutate(across(c(human, threat, benefit, pi), ~replace_na(., 0))) %>%
  filter(!(docid %in% c(451, 594))) #Filter duplicates/bad obs

#Create df for agreement column
training_agree = training %>% 
  left_join(df_four %>% select(DOCID, code_agreement) %>% group_by(DOCID) %>%
              summarize(code_agreement = max(code_agreement)), by = c("docid" = "DOCID")) 

#write.csv(training_agree, "human4_agree.csv") 

#Remove NA row and add initializer blank row
training_four_final = training[-1, ] %>% 
  add_row(docid = 0, human = 0, threat = 0, benefit = 0, pi = 0, .before = 1)

#Save as csv file for AI
#write.csv(training_four_final, "AI_goldstandard_four_v2.csv") 


#Save classification information into object (ten label)
training = tibble(value = df_ten$label, label = df_ten$theme, 
                  docid = df_ten$DOCID) 

training = training %>% 
  pivot_wider(names_from = label, values_from = value, values_fn = max) %>%
  arrange(docid) %>% mutate(across(MAIN_HUM_HUMAN:MAIN_PI, ~replace_na(., 0))) %>%
  filter(!(docid %in% c(451, 594))) #Filter duplicates/bad obs

#Create df for agreement column
training_agree = training %>% 
  left_join(df_ten %>% select(DOCID, code_agreement) %>% group_by(DOCID) %>%
              summarize(code_agreement = max(code_agreement)), by = c("docid" = "DOCID"))

#write.csv(training_agree, "human10_agree.csv") 

#Remove NA row
training_ten_final = training[-1, ] %>% 
  add_row(docid = 0, MAIN_HUM_HUMAN = 0, MAIN_HUM_REFUGE = 0, 
          MAIN_THREAT_DIS = 0, MAIN_THREAT_ECON = 0, MAIN_THREAT_VIOL = 0, 
          MAIN_THREAT_INSTAB = 0, MAIN_BEN_ECON = 0, MAIN_PI = 0, .before = 1)

#Save as csv file for AI
#write.csv(training_ten_final, "AI_goldstandard_ten_v2.csv")

rm(list = ls())
```

# ChatGPT Analysis

```{r prepare prompt and hamming loss}
api_key = ""

#Prepare object to send requests to GPT
fetch_response_openai <- function(prompt) {
  body <- toJSON(list(
    model = "gpt-4o",
    messages = list(
      list(role = "user", content = prompt),
      list(role = "system", content = "You are a research coder for newspaper articles")
    ),
    temperature = 0
  ), auto_unbox = TRUE)  # Ensures single elements are not in arrays

  response <- httr::POST(
    url = "https://api.openai.com/v1/chat/completions",
    httr::add_headers(Authorization = paste("Bearer", api_key)),
    httr::content_type_json(),
    body = body,
    encode = "json",
    httr::timeout(300)
  )

  if (response$status_code != 200) {
    stop("API request failed with status: ", response$status_code, " and message: ", content(response, "text"))
  }

  return(content(response, "parsed", type = "application/json"))
}

# hamming loss function to look at the fraction of disagreeing verdicts
hamming_loss <-
  function(standard, results) {
    if (!is.data.frame(standard) || !is.data.frame(results)) {
      stop("Both objects should be data frames.")
    }
    if (nrow(standard) != nrow(results) || ncol(standard) != ncol(results)) {
      stop("Dimensions don't match.")
    }
    samples <- nrow(standard)
    labels <- ncol(standard)
    loss <- 0
    for (i in 1:samples) {
      mismatches <- sum(standard[i, ] != results[i, ])
      loss <- loss + (mismatches/labels)
    }
    hamming_loss <- loss/samples
    return(hamming_loss)
  }

```

#First, let's do the 10 label prompts

```{r load in data, message = F}

standard = read_csv("AI_goldstandard_ten_v2.csv") %>% select(-...1)

#Limited sample...testing before full classification... %>% slice_sample(n = 500)
#Get sample rows
sample = standard  %>% select(docid) %>% arrange(docid) 

excerpts = standard %>% select(docid, text) %>% filter(docid %in% sample$docid) %>% arrange(docid)

#Prepare prompts with articles in same object
prompt_text = 
  readLines("chatgpt_prompt_ten_fewshot v7.txt", warn = FALSE, 
            encoding = "UTF-8") |>
  paste(collapse = " ") |>
  gsub("  ", " ", x = _) |>
  gsub('[\"]', "'", x = _) 

excerpts$prompt = paste(prompt_text, "<text>", excerpts$text, "</text>")

```


```{r run everything, eval = F}
#DO NOT RUN UNTIL CONFIDENT

responses_gpt10 <- pbapply::pblapply(excerpts$prompt,
                               fetch_response_openai,
                               cl = 1) #Clusters = 1

```


```{r examine and clean output}
#Extract results from JSON responses and clean them
results <-
  lapply(
    responses_gpt10,
    function(response) {
            r = gsub("json|\\n|```", "", response$choices[[1]]$message$content)
           
            r = gsub("'", '"', r)
      
            try(as_tibble(fromJSON(r)))
    }
  )

#Clean up and reformat output
clean_results = tibble()

clean_results = as_tibble(t(bind_cols(results))) %>% 
  rename(label = V1) %>% #, explanation = V2
  mutate(label = as.numeric(label), 
         theme = rep(c("Theme1", "Theme2", "Theme3", "Theme4", "Theme5", "Theme6",
                       "Theme7", "Theme8"), times = 590)) #times = sample size

#For Spanish prompts
#clean_results = as_tibble(t(bind_cols(results))) %>% 
#  rename(label = V1) %>% #, explanation = V2
#  mutate(label = as.numeric(label), 
#         theme = rep(c("Tema1", "Tema2", "Tema3", "Tema4", "Tema5", "Tema6", "Tema7",
#                       "Tema8"), times = 590)) #times = sample size

clean_results$docid = rep(excerpts$docid, each = 8)

#Pivot wider to match standard
clean_results_wide = pivot_wider(clean_results, id_cols = "docid", names_from = "theme", 
                            values_from = c("label"))

#Check the loss
#Reduce standard to size of test sample object and filter for hamming loss analysis
standard_ham = standard %>% filter(docid %in% clean_results_wide$docid) %>% 
  rename(Theme1 = MAIN_HUM_HUMAN, Theme2 = MAIN_HUM_REFUGE, Theme3 = MAIN_THREAT_DIS,
         Theme4 = MAIN_THREAT_ECON, 
         Theme5 = MAIN_THREAT_VIOL, Theme6 = MAIN_THREAT_INSTAB, Theme7 = MAIN_BEN_ECON, 
         Theme8 = MAIN_PI) %>% arrange(docid) %>% select(-c(text, docid))

clean_results_ham = clean_results_wide %>% arrange(docid) %>% select(Theme1:Theme8)

#For Spanish
#clean_results_ham = clean_results_wide %>% arrange(docid) %>% select(Tema1:Tema8)

#Rename variables
names(clean_results_ham) = c("Theme1", "Theme2", "Theme3", "Theme4", "Theme5", "Theme6",
                             "Theme7", "Theme8")

#Spanish
#names(clean_results_ham) = c("Tema1", "Tema2", "Tema3", "Tema4", "Tema5", "Tema6", "Tema7", "Tema8")

#Run hamming loss function
#Filter to sample rows
standard_test = standard %>% filter(docid %in% sample$docid) %>%
  select(MAIN_HUM_HUMAN:MAIN_PI)

names(standard_test) = c("Theme1", "Theme2", "Theme3", "Theme4", "Theme5", "Theme6",
                         "Theme7", "Theme8")

#Spanish
#names(standard_test) = c("Tema1", "Tema2", "Tema3", "Tema4", "Tema5", "Tema6", "Tema7", "Tema8")

hamming_loss(standard_test, clean_results_ham)

#write.csv(clean_results_wide, ".csv") #
```

#Now, 4 label prompts

```{r load in data for 4}
#Load in data
standard4 = read_csv("AI_goldstandard_four_v2.csv") %>% select(-...1)

#For limited sample...testing before full classification...%>% filter(docid %in% sample$docid) 
excerpts4 = standard4 %>% select(docid, text) %>%
  arrange(docid) 

#Prepare prompts with articles in same object
prompt_text4 = 
  readLines("chatgpt_prompt_four_fewshot v5.txt", warn = FALSE, 
            encoding = "UTF-8") |>
  paste(collapse = " ") |>
  gsub("  ", " ", x = _) |>
  gsub('[\"]', "'", x = _) 

excerpts4$prompt = paste(prompt_text4, "<text>", excerpts4$text, "</text>")

```


```{r run everything, eval = F}
#DO NOT RUN UNTIL CONFIDENT

responses_gpt4 <- pbapply::pblapply(excerpts4$prompt,
                               fetch_response_openai,
                               cl = 1) #Clusters = 1

```


```{r examine and clean output}
#Extract results from JSON responses and clean them
results4 <-
  lapply(
    responses_gpt4,
    function(response) {
            r = gsub("json|\\n|```", "", response$choices[[1]]$message$content)
            
            r = gsub("'", '"', r)
      
            as_tibble(fromJSON(r))
    }
  )


#Clean up and reformat output
clean_results4 = tibble()

clean_results4 = as_tibble(t(bind_cols(results4))) %>% 
  #rename(label = V1) %>% 
  mutate(label = as.numeric(V1), 
         theme = rep(c("Theme1", "Theme2", "Theme3", "Theme4"), times = 590)) 
#times = sample size

#Add in docids
clean_results4$docid = rep(excerpts4$docid, each = 4)

#Pivot wider to match standard
clean_results_wide4 = pivot_wider(clean_results4, id_cols = "docid", names_from = "theme", 
                            values_from = c("label"))

#Check the loss
#Reduce standard to size of test sample object and filter for hamming loss analysis
standard_ham4 = standard4 %>% filter(docid %in% clean_results_wide4$docid) %>% 
  rename(Theme1 = human, Theme2 = threat, Theme3 = benefit, Theme4 = pi) %>% 
  arrange(docid) %>% select(-c(text, docid))

clean_results_ham4 = clean_results_wide4 %>% arrange(docid) %>% select(Theme1:Theme4)

#Rename variables
names(clean_results_ham4) = c("Theme1", "Theme2", "Theme3", "Theme4")

#Run hamming loss function
#Filter to sample rows
standard_test4 = standard4 %>% #filter(docid %in% sample$docid) %>% 
  select(human:pi)

names(standard_test4) = c("Theme1", "Theme2", "Theme3", "Theme4")

hamming_loss(standard_test4, clean_results_ham4)

#write.csv(clean_results_wide4, ".csv") 
```


```{r check final costs}
#Pricing for GPT 4 Turbo
cost_per_token <- list(input = 10 * 10^(-6),
                       output = 30 * 10^(-6))

#Pricing for GPT 4o
cost_per_token <- list(input = 5 * 10^(-6),
                       output = 15 * 10^(-6))

#Cost of request 
costs <-
  lapply(
    responses_gpt4, #responses_gpt10
    function(response) {
      response$usage |>
        (\(.)
          .$prompt_token * cost_per_token$input +
          .$completion_tokens * cost_per_token$output
        )()
    }
  ) |>
  do.call(sum, args = _)
```

#Claude Analysis

```{r prepare prompt}
claude_key = ""

#Prepare object to send requests to GPT
fetch_response_claude <- function(prompt) {
  body <- toJSON(list(
    max_tokens = 1024,
    model = "claude-3-5-sonnet-20241022", 
    messages = list(list(role = "user", content = prompt))
  ), auto_unbox = TRUE)  # Proper JSON formatting

  response <- httr::POST(
    url = "https://api.anthropic.com/v1/messages",
    httr::add_headers(
      `x-api-key` = claude_key,
      `anthropic-version` = "2023-06-01",
      `Content-Type` = 'application/json'  # Correct header key for content type
    ),
    body = body,
    encode = "json",
    httr::content_type_json(),
    httr::timeout(300)  # Timeout set to 100 seconds
  )

  if (response$status_code != 200) {
    stop("API request failed with status: ", response$status_code, " and message: ",
         content(response, "text"))
  }

  return(content(response, "parsed", type = "application/json"))
}

```

#First, lets do the 10 label prompt

```{r load in data}
standard_claude10 = read_csv("AI_goldstandard_ten_v2.csv") %>% select(-...1)

#For limited sample...testing before full classification...%>% filter(docid %in% sample$docid)
excerpts_claude10 = standard_claude10 %>% select(docid, text) %>% arrange(docid) 

#Prepare prompts with articles in same object
prompt_text =
  readLines("claude_prompt_ten v5.txt", warn = FALSE, 
            encoding = "UTF-8") |>
  paste(collapse = " ") |>
  gsub("  ", " ", x = _) |>
  gsub('[\"]', "'", x = _)

excerpts_claude10$prompt = paste("<text>", excerpts_claude10$text, "</text>", prompt_text)

```


```{r run everything, eval = F}
#DO NOT RUN UNTIL CONFIDENT

responses_claude10 <- pbapply::pblapply(excerpts_claude10$prompt,
                               fetch_response_claude,
                               cl = 1) #Clusters = 1

```


```{r examine and clean output}
results_claude10 <-
  lapply(
    responses_claude10,
    function(response) {
            r = gsub("json|\\n|```", "", response$content[[1]]$text)
           
            r = gsub("'", '"', r)
      
            try(as_tibble(fromJSON(r)))
    }
  )

cleaned_claude10 = as_tibble(t(bind_cols(results_claude10))) %>% 
  mutate(label = as.numeric(V1), 
         theme = rep(c("Theme1", "Theme2", "Theme3", "Theme4", "Theme5", "Theme6", "Theme7",
                       "Theme8"), times = 590)) 

cleaned_claude10$docid = rep(excerpts_claude10$docid, each = 8)

#Pivot wider to match standard
clean_results_wide_claude10 = pivot_wider(cleaned_claude10, id_cols = "docid", 
                                          names_from = "theme", 
                            values_from = c("label"))

#Check the loss
#Reduce standard to size of test sample object and filter for hamming loss analysis
standard_ham_claude10 = standard_claude10 %>% 
  filter(docid %in% clean_results_wide_claude10$docid) %>% 
  rename(Theme1 = MAIN_HUM_HUMAN, Theme2 = MAIN_HUM_REFUGE, Theme3 = MAIN_THREAT_DIS, 
         Theme4 = MAIN_THREAT_ECON, 
         Theme5 = MAIN_THREAT_VIOL, Theme6 = MAIN_THREAT_INSTAB, Theme7 = MAIN_BEN_ECON, 
         Theme8 = MAIN_PI) %>% arrange(docid) %>% select(-c(text, docid))

clean_results_ham_claude10 = clean_results_wide_claude10 %>% arrange(docid) %>%
  select(Theme1:Theme8)

#Rename variables
names(clean_results_ham_claude10) = c("Theme1", "Theme2", "Theme3", "Theme4", "Theme5", "Theme6",
                                      "Theme7", "Theme8")


#Filter to sample rows...not going to change anything for final run because it's using the pop.
standard_test_claude10 = standard_claude10 %>% 
  #filter(docid %in% sample$docid) %>% 
  select(MAIN_HUM_HUMAN:MAIN_PI)

names(standard_test_claude10) = c("Theme1", "Theme2", "Theme3", "Theme4", "Theme5", "Theme6",
                                  "Theme7", "Theme8")

#Run hamming loss function
hamming_loss(standard_test_claude10, clean_results_ham_claude10)

#Sonnet
#write.csv(clean_results_wide_claude10, ".csv") 
```

# Now, 4 label Claude

```{r load in data}
standard_claude4 = read_csv("AI_goldstandard_four_v2.csv") %>% select(-...1)

#For limited sample...testing before full classification...filter(docid %in% sample$docid) %>%
excerpts_claude4 = standard_claude4 %>% select(docid, text) %>% 
   arrange(docid) #Seed includes initializer row, so no need to add it in

#Prepare prompts with articles in same object
prompt_text =
  readLines("claude_prompt_four v5.txt", warn = FALSE, 
            encoding = "UTF-8") |>
  paste(collapse = " ") |>
  gsub("  ", " ", x = _) |>
  gsub('[\"]', "'", x = _)

excerpts_claude4$prompt = paste("<text>", excerpts_claude4$text, "</text>", prompt_text)

```


```{r run everything, eval = F}
#DO NOT RUN UNTIL CONFIDENT

responses_claude4 <- pbapply::pblapply(excerpts_claude4$prompt,
                               fetch_response_claude,
                               cl = 1) #Clusters = 1

```


```{r examine and clean output}

results_claude4 <-
  lapply(
    responses_claude4,
    function(response) {
            r = gsub("json|\\n|```", "", response$content[[1]]$text)
           
            r = gsub("'", '"', r)
      
            try(as_tibble(fromJSON(r)))
    }
  )

cleaned_claude4 = as_tibble(t(bind_cols(results_claude4))) %>% 
  mutate(label = as.numeric(V1), 
         theme = rep(c("Theme1", "Theme2", "Theme3", "Theme4"), times = 590)) 

#Add in docids
cleaned_claude4$docid = rep(excerpts_claude4$docid, each = 4)

#Pivot wider to match standard
clean_results_wide_claude4 = pivot_wider(cleaned_claude4, id_cols = "docid", 
                                         names_from = "theme", 
                            values_from = c("label")) #, "explanation"

#Check the loss
#Reduce standard to size of test sample object and filter for hamming loss analysis
standard_ham_claude4 = standard_claude4 %>% 
  #filter(docid %in% clean_results_wide_claude4$docid) %>% 
  rename(Theme1 = human, Theme2 = threat, Theme3 = benefit, 
         Theme4 = pi) %>% arrange(docid) %>% select(-c(text, docid))

clean_results_ham_claude4 = clean_results_wide_claude4 %>% arrange(docid) %>%
  select(Theme1:Theme4)

#Rename variables
names(clean_results_ham_claude4) = c("Theme1", "Theme2", "Theme3", "Theme4")

#Run hamming loss function
#Filter to sample rows
standard_test_claude4 = standard_claude4 %>% 
  #filter(docid %in% sample$docid) %>%
  select(human:pi)

names(standard_test_claude4) = c("Theme1", "Theme2", "Theme3", "Theme4")

hamming_loss(standard_test_claude4, clean_results_ham_claude4)

#Sonnet
#write.csv(clean_results_wide_claude4, ".csv") 
```


```{r check final costs}
#Pricing for Claude Sonnet 3 and 3.5
cost_per_token_claude <- list(input = 3 * 10^(-6),
                       output = 15 * 10^(-6))

#Pricing for Claude Opus
#cost_per_token <- list(input = 15 * 10^(-6),
#                       output = 75 * 10^(-6))

#Cost of request 
costs <-
  lapply(
    responses_claude4, #responses_claude10
    function(response) {
      response$usage |>
        (\(.)
         .$input_tokens * cost_per_token_claude$input +
           .$output_tokens * cost_per_token_claude$output
        )()
    }
  ) |>
  do.call(sum, args = _)
```

#Randomize order of prompts

```{r create prompt pieces for random assignment}
#First, save in pieces of each prompt

#10 label
#Zero shot
z8_lab1 = "Theme1: Humanitarianism, empathy, and perspective-taking. Does the article portray immigrants as vulnerable victims of circumstance or focus on their poverty, victimhood, and lack of choice? Does the article attempt to make the audience see life from an immigrant’s point of view? Does it provoke sympathy for immigrants?"
  
z8_lab2 = "Theme2: Persecution, poverty, and push factors. Does the article mention that some immigrants are refugees fleeing persecution or extreme economic deprivation?"
  
z8_lab3 = "Theme3: Health issues and threat of disease. Does the article relate issues of health and disease spread to immigration?"
  
z8_lab4 = "Theme4: Economic issues, resource scarcity, and threat. Does the article mention how immigration may lead to worse economic outcomes, such as job loss, lower salaries, or less access to public services for citizens?"

z8_lab5 = "Theme5: Issues of crime, violence, and illegal activities. Does the article mention how immigration may lead to increases in violence, crime, or physical insecurity? Does the article associate immigrants with crime, violence, or illicit activities?"

z8_lab6 = 'Theme6: General instability and threat. Does the connect political or societal instability to immigration? Does it use the size of immigration flows to provoke worry or fear (for example, "hordes of immigrants" or "massive immigration flows")?'

z8_lab7 = "Theme7: Economic benefit. Does the article mention how immigration may improve economic conditions?"

z8_lab8 = "Theme8: Immigration policies and integration. Does the article bring up the relationship between immigration policies or government actions and immigrant integration?"

#One shot
o8_lab1 = 'Theme1: Humanitarianism, empathy, and perspective-taking. Does the article portray immigrants as vulnerable victims of circumstance or focus on their poverty, victimhood, and lack of choice? Does the article attempt to make the audience see life from an immigrant’s point of view? Does it provoke sympathy for immigrants?

Example: "A pesar de que las víctimas que son residentes han sido maltratadas y tienen procesos muy duros, las que están en el exterior son doblemente victimizadas porque se han tenido que ir a otro país y vivir todo el proceso del migrante, adicional al proceso de ser víctima vulnerable."'
  
o8_lab2 = 'Theme2: Persecution, poverty, and push factors. Does the article mention that some immigrants are refugees fleeing persecution or extreme economic deprivation?

Example: "El drama de más de 100 migrantes, refugiados en parque Las Banderas. Muchos de estos migrantes habían clamado por ayuda, sobre todo, porque había niños y adultos mayores con síntomas de gripa, debido a la intensas lluvias de las últimas semanas."'
  
o8_lab3 = 'Theme3: Health issues and threat of disease. Does the article relate issues of health and disease spread to immigration?

Example: "El presidente de Colombia Iván Duque anunció que se cerrarán los, siete pasos fronterizos con Venezuela, con el objetivo de tratar de detener la propagación del coronavirus en el país."'
  
o8_lab4 = 'Theme4: Economic issues, resource scarcity, and threat. Does the article mention how immigration may lead to worse economic outcomes, such as job loss, lower salaries, or less access to public services for citizens?

Example: "Desde que llegó hace un mes, reparte las horas entre buscar empleo y buscar comida. "No hay trabajo para los cucuteños menos para el venezolano", dice este exsargento del Ejército."'

o8_lab5 = 'Theme5: Issues of crime, violence, and illegal activities. Does the article mention how immigration may lead to increases in violence, crime, or physical insecurity? Does the article associate immigrants with crime, violence, or illicit activities?

Example: "Es cierto que grupos criminales se estén aprovechando de estos venezolanos. Resulta que en el año 2016 se han capturado, por diferentes delitos, 242 ciudadanos de Venezuela en el área metropolitana de Cúcuta."'

o8_lab6 = 'Theme6: General instability and threat. Does the connect political or societal instability to immigration? Does it use the size of immigration flows to provoke worry or fear (for example, "hordes of immigrants" or "massive immigration flows")?

Example: "El creciente flujo de refugiados hacia Colombia muestra el caos humanitario que se avecina. Ante la tragedia, los mandatarios de la región han resuelto enterrar la cabeza como avestruces."'

o8_lab7 = 'Theme7: Economic benefit. Does the article mention how immigration may improve economic conditions?

Example: "Si hay alguna característica que uno puede buscar en un inmigrante es, que es gente aspiracional, gente que busca algo mejor. Van a trabajar, a sacrificarse, a luchar."'

o8_lab8 = 'Theme8: Immigration policies and integration. Does the article bring up the relationship between immigration policies or government actions and immigrant integration?

Example: "Sus políticas de integración, salud y educación han sido pioneras en la región, añadió el embajador. Colombia sigue necesitando mucho apoyo de la cooperación internacional para atender a los migrantes."'

#Few shot
f8_lab1 = 'Theme1: Humanitarianism, empathy, and perspective-taking. Does the article portray immigrants as vulnerable victims of circumstance or focus on their poverty, victimhood, and lack of choice? Does the article attempt to make the audience see life from an immigrant’s point of view? Does it provoke sympathy for immigrants?

Example: "A pesar de que las víctimas que son residentes han sido maltratadas y tienen procesos muy duros, las que están en el exterior son doblemente victimizadas porque se han tenido que ir a otro país y vivir todo el proceso del migrante, adicional al proceso de ser víctima vulnerable."

Example: "También tuvo un recuerdo en su mensaje de Navidad para los desplazados, los emigrantes y refugiados, y los que hoy son objeto de la trata de personas y lamentó que muchos pueblos sufren por las ambiciones económicas de unos pocos y la avaricia voraz del dios dinero que lleva a la esclavitud."

Example: "El papa aprovechó para recordar “las prolongadas penalidades y angustias” de la crisis humanitaria de Venezuela, agravadas por la pandemia, así como a “todos aquellos que han dejado el país en busca de mejores condiciones de vida”, al referirse a los millones de venezolanos que han tenido que emigrar hacia otras naciones."'
  
f8_lab2 = 'Theme2: Persecution, poverty, and push factors. Does the article mention that some immigrants are refugees fleeing persecution or extreme economic deprivation?

Example: "El drama de más de 100 migrantes, refugiados en parque Las Banderas. Muchos de estos migrantes habían clamado por ayuda, sobre todo, porque había niños y adultos mayores con síntomas de gripa, debido a la intensas lluvias de las últimas semanas."

Example: "El 80% de esas personas refugiadas está en una situación de refugio de largo plazo, es decir que en más de cinco años no han podido volver a su país de origen, y esto se da o porque el conflicto sigue o porque hay explosiones de otros conflictos."

Example: "Más de un millón de personas han huido del país en la última década, el 90 por ciento en los últimos cuatro años. La desesperación, el empobrecimiento y la irritación de los venezolanos está creciendo aceleradamente, ocurren brotes espontáneos de violencia todos los días. Existe una anarquía que deja la sensación de que no hay gobierno."'
  
f8_lab3 = 'Theme3: Health issues and threat of disease. Does the article relate issues of health and disease spread to immigration?

Example: "El presidente de Colombia Iván Duque anunció que se cerrarán los, siete pasos fronterizos con Venezuela, con el objetivo de tratar de detener la propagación del coronavirus en el país."

Example: "Los médicos descubrieron entonces que los migrantes se contagiaban en los ríos, donde caen las heces contaminadas con cercarias de las aves migratorias que transitan por el Darién."

Example: "Esto los tiene en alerta roja con una ocupación hospitalaria en las unidades de cuidados intensivos que llega al 98 por ciento y que se agravaría si los migrantes venezolanos continúan su paso sin control."'
  
f8_lab4 = 'Theme4: Economic issues, resource scarcity, and threat. Does the article mention how immigration may lead to worse economic outcomes, such as job loss, lower salaries, or less access to public services for citizens?

Example: "Desde que llegó hace un mes, reparte las horas entre buscar empleo y buscar comida. "No hay trabajo para los cucuteños menos para el venezolano", dice este exsargento del Ejército."

Example: "Aunque oficialmente no se ha reconocido un monto exacto, se calcula que cada paciente crónico extranjero atendido le cuesta a Colombia entre 200 y 220 millones de pesos al año.  El Ministro recalcó que estas cifras son parciales y no dimensionan el gasto total para el país. Por eso pidió a todas las entidades territoriales y a las EPS, a través de una circular de febrero pasado, detallar los costos que ha generado este año la atención a extranjeros."

Example: "¿El problema es ver a los refugiados como una carga para los Estados? Así es. En el caso de Ecuador, este lunes el Gobierno dio una rueda de prensa en la que explicaban que los 56.000 refugiados colombianos le cuestan al Estado 30 millones de dólares."'

f8_lab5 = 'Theme5: Issues of crime, violence, and illegal activities. Does the article mention how immigration may lead to increases in violence, crime, or physical insecurity? Does the article associate immigrants with crime, violence, or illicit activities?

Example: "Es cierto que grupos criminales se estén aprovechando de estos venezolanos. Resulta que en el año 2016 se han capturado, por diferentes delitos, 242 ciudadanos de Venezuela en el área metropolitana de Cúcuta." 

Example: "Teniendo en cuenta que el ingreso de venezolanos ilegales a la ciudad ha sido relacionado con el incremento de la inseguridad, el Mandatario dijo que solicitaron una unidad especial de Migración Colombia para realizar algunas deportaciones."

Example: "Lo que ha representado problemas de desorden social en la región fronteriza, casos de trata de personas, victimización violenta, extorsión y despojos, el desarrollo de toda una economía informal en torno a la masiva migración y un importante ejército de reserva de mano de obra barata para la economía legal, informal e ilegal, según recoge Pares en su informe."'

f8_lab6 = 'Theme6: General instability and threat. Does the connect political or societal instability to immigration? Does it use the size of immigration flows to provoke worry or fear (for example, "hordes of immigrants" or "massive immigration flows")?

Example: "El creciente flujo de refugiados hacia Colombia muestra el caos humanitario que se avecina. Ante la tragedia, los mandatarios de la región han resuelto enterrar la cabeza como avestruces."

Example: "La ciudad, de 350.000 habitantes, está colapsada. Y si no fuera por las remesas de su hermano, Tilus y su familia estarían en la calle, como otros migrantes."

Example: "La crisis no termina y estamos tratando con consecuencias, de otra serie de problemas políticos que si no se solucionan este flujo, no logrará controlarse."'

f8_lab7 = 'Theme7: Economic benefit. Does the article mention how immigration may improve economic conditions?

Example: "Si hay alguna característica que uno puede buscar en un inmigrante es, que es gente aspiracional, gente que busca algo mejor. Van a trabajar, a sacrificarse, a luchar."

Example: "Colombia tendrá que destinar entre 0,23 y 0,41% de su PIB en el corto plazo para atender a los migrantes venezolanos que huyen de crisis en su país, aunque bien gestionada la ola migratoria puede darle réditos económicos a mediano y largo plazo."

Example: "Pero si esos refugiados estuvieran legalizados, podrían pagar impuestos, trabajar, y aportar a la economía de ese país."'

f8_lab8 = 'Theme8: Immigration policies and integration. Does the article bring up the relationship between immigration policies or government actions and immigrant integration?

Example: "Sus políticas de integración, salud y educación han sido pioneras en la región, añadió el embajador. Colombia sigue necesitando mucho apoyo de la cooperación internacional para atender a los migrantes."

Example: "Una de esas medidas, acordadas el pasado 4 de agosto, fue la creación de una cédula fronteriza. El documento, que deben portar tanto los ciudadanos venezolanos como colombianos que residen en la frontera, tiene contenida información fundamental de las actividades que desarrollan y los motivos de su paso entre ambos países."

Example: "Los venezolanos que residen en Colombia y cuentan con el Permiso por Protección Temporal (PPT) podrán obtener su licencia de conducción sin ningún problema. Esto bajo una resolución del Ministerio de Transporte expedida en los últimos días del mandato de Iván Duque. Todo migrante venezolano que tenga con el PPT lo podrá utilizar para realizar trámites asociados con la oficina de tránsito, es decir, podrán iniciar un proceso para solicitar su licencia de conducción y así transitar en vehículos de manera legal por el país."'

#4 label
#Zero shot
z4_lab1 = "Theme1: Humanitarianism, empathy, and perspective-taking. Does the article portray immigrants as vulnerable victims of circumstance or focus on their poverty, victimhood, and lack of choice? Does the article attempt to make the audience see life from an immigrant’s point of view, encouraging further thinking and reflection on the issue? Does it provoke sympathy for immigrants? Does the article mention that some immigrants are refugees fleeing persecution or extreme economic deprivation?"
  
z4_lab2 = "Theme2: Economic, disease, and violent threats. Does the article relate issues of health and disease spread to immigration? Does the article mention how immigration may lead to worse economic outcomes, such as job loss, lower salaries, or less access to public services for citizens? Does the article mention how immigration may lead to increases in violence, crime, or physical insecurity? Does the article associate immigrants with crime, violence, or illicit activities? Does the connect political or societal instability to immigration? Does it use the size of immigration flows to provoke worry or fear (for example, language like 'hordes of immigrants' or 'massive immigration flows')? "
  
z4_lab3 = "Theme3: Economic benefits. Does the article mention how immigration may improve economic conditions?"
  
z4_lab4 = "Theme4: Immigration policies and integration. Does the article bring up the relationship between immigration policies or government actions and immigrant integration?"

#One shot
o4_lab1 = "Theme1: Humanitarianism, empathy, and perspective-taking. Does the article portray immigrants as vulnerable victims of circumstance or focus on their poverty, victimhood, and lack of choice? Does the article attempt to make the audience see life from an immigrant’s point of view, encouraging further thinking and reflection on the issue? Does it provoke sympathy for immigrants? Does the article mention that some immigrants are refugees fleeing persecution or extreme economic deprivation? 

Example: 'El drama de más de 100 migrantes, refugiados en parque Las Banderas. Muchos de estos migrantes habían clamado por ayuda, sobre todo, porque había niños y adultos mayores con síntomas de gripa, debido a la intensas lluvias de las últimas semanas.'"
  
o4_lab2 = "Theme2: Economic, disease, and violent threats. Does the article relate issues of health and disease spread to immigration? Does the article mention how immigration may lead to worse economic outcomes, such as job loss, lower salaries, or less access to public services for citizens? Does the article mention how immigration may lead to increases in violence, crime, or physical insecurity? Does the article associate immigrants with crime, violence, or illicit activities? Does the connect political or societal instability to immigration? Does it use the size of immigration flows to provoke worry or fear (for example, language like 'hordes of immigrants' or 'massive immigration flows')? 

Example: 'Aunque oficialmente no se ha reconocido un monto exacto, se calcula que cada paciente crónico extranjero atendido le cuesta a Colombia entre 200 y 220 millones de pesos al año. El Ministro recalcó que estas cifras son parciales y no dimensionan el gasto total para el país. Por eso pidió a todas las entidades territoriales y a las EPS, a través de una circular de febrero pasado, detallar los costos que ha generado este año la atención a extranjeros.'"
  
o4_lab3 = "Theme3: Economic benefits. Does the article mention how immigration may improve economic conditions? 

Example: 'Si hay alguna característica que uno puede buscar en un inmigrante es, que es gente aspiracional, gente que busca algo mejor. Van a trabajar, a, sacrificarse, a luchar.'"
  
o4_lab4 = "Theme4: Immigration policies and integration. Does the article bring up the relationship between immigration policies or government actions and immigrant integration? 

Example: 'Sus políticas de integración, salud y educación han sido pioneras en la región, añadió el embajador. Colombia sigue necesitando mucho apoyo de la cooperación internacional para atender a los migrantes.'"

#Few shot
f4_lab1 = 'Theme1: Humanitarianism, empathy, and perspective-taking. Does the article portray immigrants as vulnerable victims of circumstance or focus on their poverty, victimhood, and lack of choice? Does the article attempt to make the audience see life from an immigrant’s point of view, encouraging further thinking and reflection on the issue? Does it provoke sympathy for immigrants? Does the article mention that some immigrants are refugees fleeing persecution or extreme economic deprivation? 

Example: "El drama de más de 100 migrantes, refugiados en parque Las Banderas. Muchos de estos migrantes habían clamado por ayuda, sobre todo, porque había niños y adultos mayores con síntomas de gripa, debido a la intensas lluvias de las últimas semanas."

Example: "El papa aprovechó para recordar “las prolongadas penalidades y angustias” de la crisis humanitaria de Venezuela, agravadas por la pandemia, así como a “todos aquellos que han dejado el país en busca de mejores condiciones de vida”, al referirse a los millones de venezolanos que han tenido que emigrar hacia otras naciones."

Example: "Más de un millón de personas han huido del país en la última década, el 90 por ciento en los últimos cuatro años. La desesperación, el empobrecimiento y la irritación de los venezolanos está creciendo aceleradamente, ocurren brotes espontáneos de violencia todos los días. Existe una anarquía que deja la sensación de que no hay gobierno."'
  
f4_lab2 = 'Theme2: Economic, disease, and violent threats. Does the article relate issues of health and disease spread to immigration? Does the article mention how immigration may lead to worse economic outcomes, such as job loss, lower salaries, or less access to public services for citizens? Does the article mention how immigration may lead to increases in violence, crime, or physical insecurity? Does the article associate immigrants with crime, violence, or illicit activities? Does the connect political or societal instability to immigration? Does it use the size of immigration flows to provoke worry or fear (for example, language like "hordes of immigrants" or "massive immigration flows")? 

Example: "Aunque oficialmente no se ha reconocido un monto exacto, se calcula que cada paciente crónico extranjero atendido le cuesta a Colombia entre 200 y 220 millones de pesos al año. El Ministro recalcó que estas cifras son parciales y no dimensionan el gasto total para el país. Por eso pidió a todas las entidades territoriales y a las EPS, a través de una circular de febrero pasado, detallar los costos que ha generado este año la atención a extranjeros."

Example: "El presidente de Colombia Iván Duque anunció que se cerrarán los, siete pasos fronterizos con Venezuela, con el objetivo de tratar de detener la propagación del coronavirus en el país."

Example: "Lo que ha representado problemas de desorden social en la región fronteriza, casos de trata de personas, victimización violenta, extorsión y despojos, el desarrollo de toda una economía informal en torno a la masiva migración y un importante ejército de reserva de mano de obra barata para la economía legal, informal e ilegal, según recoge Pares en su informe."'
  
f4_lab3 = 'Theme3: Economic benefits. Does the article mention how immigration may improve economic conditions? 

Example: "Si hay alguna característica que uno puede buscar en un inmigrante es, que es gente aspiracional, gente que busca algo mejor. Van a trabajar, a, sacrificarse, a luchar."

Example: "Colombia tendrá que destinar entre 0,23 y 0,41% de su PIB en el corto plazo para atender a los migrantes venezolanos que huyen de crisis en su país, aunque bien gestionada la ola migratoria puede darle réditos económicos a mediano y largo plazo."

Example: "Pero si esos refugiados estuvieran legalizados, podrían pagar impuestos, trabajar, y aportar a la economía de ese país."
'
  
f4_lab4 = 'Theme4: Immigration policies and integration. Does the article bring up the relationship between immigration policies or government actions and immigrant integration? 

Example: "Sus políticas de integración, salud y educación han sido pioneras en la región, añadió el embajador. Colombia sigue necesitando mucho apoyo de la cooperación internacional para atender a los migrantes."

Example: "Una de esas medidas, acordadas el pasado 4 de agosto, fue la creación de una cédula fronteriza. El documento, que deben portar tanto los ciudadanos venezolanos como colombianos que residen en la frontera, tiene contenida información fundamental de las actividades que desarrollan y los motivos de su paso entre ambos países."

Example: "Los venezolanos que residen en Colombia y cuentan con el Permiso por Protección Temporal (PPT) podrán obtener su licencia de conducción sin ningún problema. Esto bajo una resolución del Ministerio de Transporte expedida en los últimos días del mandato de Iván Duque. Todo migrante venezolano que tenga con el PPT lo podrá utilizar para realizar trámites asociados con la oficina de tránsito, es decir, podrán iniciar un proceso para solicitar su licencia de conducción y así transitar en vehículos de manera legal por el país."'

#Pre and post text
#ChatGPT
#zero
pre_zero = "Consider the following themes that may be present in newspaper articles:"

post_zero = 'I will provide you with a newspaper article, and you will determine whether it matches any of the above themes. Label each theme with a 1 if it occurs in the article and a 0 if not. For example, if only theme 3 is mentioned, your response should be: {"Theme1": 0, "Theme2": 0, "Theme3": 1, ...}. If the article is not primarily concerned with immigration, code all themes as 0s. 

Your response should be structured like this: [{"Theme1": 0, "Theme2": 0, "Theme3": 1, ...}] 

Do not include any output other than the JSON structured response, and your responses should be in English. The newspaper article is below:'

#one
pre_one = "Consider the following themes that may be present in newspaper articles. For each theme, I provide an explanation and then an example text embodying that theme."

post_one = 'I will provide you with a newspaper article, and you will determine whether it matches any of the above themes. Label each theme with a 1 if it occurs in the article and a 0 if not. For example, if only theme 3 is mentioned, your response should be: {"Theme1": 0, "Theme2": 0, "Theme3": 1, ...}. If the article is not primarily concerned with immigration, code all themes as 0s. 

Your response should be structured like this: [{"Theme1": 0, "Theme2": 0, "Theme3": 1, ...}] 

Do not include any output other than the JSON structured response, and your responses should be in English. The newspaper article is below:'

#few 
pre_few = 'Consider the following themes that may be present in newspaper articles. For each theme, I provide an explanation and then a few example texts embodying that theme.'

post_few = 'I will provide you with a newspaper article, and you will determine whether it matches any of the above themes. Label each theme with a 1 if it occurs in the article and a 0 if not. For example, if only theme 3 is mentioned, your response should be: {"Theme1": 0, "Theme2": 0, "Theme3": 1, ...}. If the article is not primarily concerned with immigration, code all themes as 0s. 

Your response should be structured like this: [{"Theme1": 0, "Theme2": 0, "Theme3": 1, ...}] 

Do not include any output other than the JSON structured response, and your responses should be in English. The newspaper article is below: '

#Claude 
#zero
clpre_zero = "Consider the following themes that may be present in the above newspaper article:"

clpost_zero = 'For the above newspaper article, determine if it matches any of these themes. Label each theme with a 1 if it occurs in the article and a 0 if not. For example, if only theme 3 is mentioned, your response should be: {"Theme1": 0, "Theme2": 0, "Theme3": 1, ...}. If the article is not primarily concerned with immigration, code all themes as 0s. 

Your response should be structured like this: [{"Theme1": 0, "Theme2": 0, "Theme3": 1, ...}] 

Do not include any output other than the JSON structured response. Ensure your output is in English.'

#one
clpre_one = "Consider the following themes that may be present in the above newspaper article. For each theme, I provide an explanation and then an example text embodying that theme."

clpost_one = 'For the above newspaper article, determine if it matches any of these themes. Label each theme with a 1 if it occurs in the article and a 0 if not. For example, if only theme 3 is mentioned, your response should be: {"Theme1": 0, "Theme2": 0, "Theme3": 1, ...}. If the article is not primarily concerned with immigration, code all themes as 0s. 

Your response should be structured like this: [{"Theme1": 0, "Theme2": 0, "Theme3": 1, ...}] 

Do not include any output other than the JSON structured response. Ensure your output is in English.'

#few 
clpre_few = 'Consider the following themes that may be present in the above newspaper article. For each theme, I provide an explanation and then a few example texts embodying that theme.'

clpost_few = 'For the above newspaper article, determine if it matches any of these themes. Label each theme with a 1 if it occurs in the article and a 0 if not. For example, if only theme 3 is mentioned, your response should be: {"Theme1": 0, "Theme2": 0, "Theme3": 1, ...}. If the article is not primarily concerned with immigration, code all themes as 0s. 

Your response should be structured like this: [{"Theme1": 0, "Theme2": 0, "Theme3": 1, ...}] 

Do not include any output other than the JSON structured response. Ensure your output is in English.'

```

#First, let's do the 10 label prompts

```{r load in data}
#Set seed for randomization
set.seed(1221) 

standard = read_csv("AI_goldstandard_ten_v2.csv") %>% select(-...1)

excerpts = standard %>% select(docid, text) %>% arrange(docid)

# Prepare prompts with articles in same object
# Define labels and pre/post texts
zero_labels <- c(z8_lab1, z8_lab2, z8_lab3, z8_lab4, z8_lab5, z8_lab6, z8_lab7, z8_lab8)
one_labels <- c(o8_lab1, o8_lab2, o8_lab3, o8_lab4, o8_lab5, o8_lab6, o8_lab7, o8_lab8)
few_labels <- c(f8_lab1, f8_lab2, f8_lab3, f8_lab4, f8_lab5, f8_lab6, f8_lab7, f8_lab8)

# Function to generate randomized prompts
generate_prompt <- function(labels, pre_text, post_text, text) {
  randomized_labels <- sample(labels)
  rand_text <- paste(randomized_labels, collapse = " ")
  prompt_text <- paste0(pre_text, " ", rand_text, " ", post_text, " <text> ", text, " </text>")
  return(prompt_text)
}

# Generate randomized prompts for each row
excerpts <- excerpts %>%
  mutate(
    prompt_zero = map_chr(text, ~generate_prompt(zero_labels, pre_zero, post_zero, .x)),
    prompt_one = map_chr(text, ~generate_prompt(one_labels, pre_one, post_one, .x)),
    prompt_few = map_chr(text, ~generate_prompt(few_labels, pre_few, post_few, .x))
  )

```


```{r run everything, eval = F}
#Get feedback
responses_gpt10_zero <- pbapply::pblapply(excerpts$prompt_zero,
                               fetch_response_openai,
                               cl = 1) #Clusters = 1
```


```{r run everything, eval = F}
#Get feedback
responses_gpt10_one <- pbapply::pblapply(excerpts$prompt_one,
                               fetch_response_openai,
                               cl = 1) #Clusters = 1
```


```{r run everything, eval = F}
#Get feedback
responses_gpt10_few <- pbapply::pblapply(excerpts$prompt_few,
                               fetch_response_openai,
                               cl = 1) #Clusters = 1
```


```{r}
#Extract results from JSON responses and clean them
results <-
  lapply(
    responses_gpt10_few,
    function(response) {
            r = gsub("json|\\n|```", "", response$choices[[1]]$message$content)
           
            r = gsub("'", '"', r)
      
            as_tibble(fromJSON(r))
    }
  )

# Reorder columns to be the same across DFs
common_columns <- c("Theme1", "Theme2", "Theme3", "Theme4", "Theme5", "Theme6", "Theme7", "Theme8")

# Reorder columns in each data frame based on the common set
reorder_columns <- function(df, columns) {
  if (is.null(df)) return(NULL)
  df <- df %>%
    select(all_of(columns)) %>%
    add_column(!!!setdiff(columns, colnames(df)), .after = last_col())
  return(df)
}

results <- lapply(results, reorder_columns, columns = common_columns)

#Clean up and reformat output
clean_results = tibble()

clean_results = as_tibble(t(bind_cols(results))) %>% 
  rename(label = V1) %>% #, explanation = V2
  mutate(label = as.numeric(label), 
         theme = rep(c("Theme1", "Theme2", "Theme3", "Theme4", "Theme5", "Theme6", "Theme7",
                       "Theme8"), times = 590)) #times = sample size

clean_results$docid = rep(excerpts$docid, each = 8)

#Pivot wider to match standard
clean_results_wide = pivot_wider(clean_results, id_cols = "docid", names_from = "theme", 
                            values_from = c("label"))

#Check the loss
#Reduce standard to size of test sample object and filter for hamming loss analysis
standard_ham = standard %>% filter(docid %in% clean_results_wide$docid) %>% 
  rename(Theme1 = MAIN_HUM_HUMAN, Theme2 = MAIN_HUM_REFUGE, Theme3 = MAIN_THREAT_DIS, 
         Theme4 = MAIN_THREAT_ECON, 
         Theme5 = MAIN_THREAT_VIOL, Theme6 = MAIN_THREAT_INSTAB, Theme7 = MAIN_BEN_ECON, 
         Theme8 = MAIN_PI) %>% arrange(docid) %>% select(-c(text, docid))

clean_results_ham = clean_results_wide %>% arrange(docid) %>% select(Theme1:Theme8)

#Rename variables
names(clean_results_ham) = c("Theme1", "Theme2", "Theme3", "Theme4", "Theme5", "Theme6", "Theme7",
                             "Theme8")

#Run hamming loss function
#Filter to sample rows
standard_test = standard %>% select(MAIN_HUM_HUMAN:MAIN_PI)

names(standard_test) = c("Theme1", "Theme2", "Theme3", "Theme4", "Theme5", "Theme6", "Theme7",
                             "Theme8")

hamming_loss(standard_test, clean_results_ham)

#write.csv(clean_results_wide, ".csv") 
```

#Now, 4 label prompts

```{r load in data for 4}
set.seed(1221)

#Load in data
standard4 = read_csv("AI_goldstandard_four_v2.csv") %>% select(-...1)

#For limited sample...testing before full classification...%>% filter(docid %in% sample$docid) 
excerpts4 = standard4 %>% select(docid, text) %>%
  arrange(docid) #Seed includes initializer row, so no need to add it in

# Prepare prompts with articles in same object
zero_labels4 <- c(z4_lab1, z4_lab2, z4_lab3, z4_lab4)
one_labels4 <- c(o4_lab1, o4_lab2, o4_lab3, o4_lab4)
few_labels4 <- c(f4_lab1, f4_lab2, f4_lab3, f4_lab4)

# Function to generate randomized prompts
generate_prompt <- function(labels, pre_text, post_text, text) {
  randomized_labels <- sample(labels)
  rand_text <- paste(randomized_labels, collapse = " ")
  prompt_text <- paste0(pre_text, " ", rand_text, " ", post_text, " <text> ", text, " </text>")
  return(prompt_text)
}

# Generate randomized prompts for each row
excerpts4 <- excerpts4 %>%
  mutate(
    prompt_zero = map_chr(text, ~generate_prompt(zero_labels4, pre_zero, post_zero, .x)),
    prompt_one = map_chr(text, ~generate_prompt(one_labels4, pre_one, post_one, .x)),
    prompt_few = map_chr(text, ~generate_prompt(few_labels4, pre_few, post_few, .x))
  )

```


```{r run everything, eval = F}
#Get feedback
responses_gpt4_zero <- pbapply::pblapply(excerpts4$prompt_zero,
                               fetch_response_openai,
                               cl = 1) #Clusters = 1

```


```{r run everything, eval = F}
#Get feedback
responses_gpt4_one <- pbapply::pblapply(excerpts4$prompt_one,
                               fetch_response_openai,
                               cl = 1) #Clusters = 1

```


```{r run everything, eval = F}
#Get feedback
responses_gpt4_few <- pbapply::pblapply(excerpts4$prompt_few,
                               fetch_response_openai,
                               cl = 1) #Clusters = 1

```


```{r examine and clean output}
#Extract results from JSON responses and clean them
results4 <-
  lapply(
    responses_gpt4_few,
    function(response) {
            r = gsub("json|\\n|```", "", response$choices[[1]]$message$content)
            
            r = gsub("'", '"', r)
      
            as_tibble(fromJSON(r))
    }
  )

# Reorder columns to be the same across DFs
common_columns4 <- c("Theme1", "Theme2", "Theme3", "Theme4")

# Reorder columns in each data frame based on the common set
reorder_columns <- function(df, columns) {
  if (is.null(df)) return(NULL)
  df <- df %>%
    select(all_of(columns)) %>%
    add_column(!!!setdiff(columns, colnames(df)), .after = last_col())
  return(df)
}

results4 <- lapply(results4, reorder_columns, columns = common_columns4)

#Clean up and reformat output
clean_results4 = tibble()

clean_results4 = as_tibble(t(bind_cols(results4))) %>% 
  #rename(label = V1) %>% 
  mutate(label = as.numeric(V1), 
         theme = rep(c("Theme1", "Theme2", "Theme3", "Theme4"), times = 590)) #times = sample size

#Add in docids
clean_results4$docid = rep(excerpts4$docid, each = 4)

#Pivot wider to match standard
clean_results_wide4 = pivot_wider(clean_results4, id_cols = "docid", names_from = "theme", 
                            values_from = c("label"))

#Check the loss
#Reduce standard to size of test sample object and filter for hamming loss analysis
standard_ham4 = standard4 %>% filter(docid %in% clean_results_wide4$docid) %>% 
  rename(Theme1 = human, Theme2 = threat, Theme3 = benefit, Theme4 = pi) %>% 
  arrange(docid) %>% select(-c(text, docid))

clean_results_ham4 = clean_results_wide4 %>% arrange(docid) %>% select(Theme1:Theme4)

#Rename variables
names(clean_results_ham4) = c("Theme1", "Theme2", "Theme3", "Theme4")

#Run hamming loss function
#Filter to sample rows
standard_test4 = standard4 %>% #filter(docid %in% sample$docid) %>% 
  select(human:pi)

names(standard_test4) = c("Theme1", "Theme2", "Theme3", "Theme4")

hamming_loss(standard_test4, clean_results_ham4)

#save classifications for understanding of temporal changes
#write.csv(clean_results_wide4, ".csv") 
```

#Random order with Claude

#First, lets do the 10 label prompt

```{r load in data}
set.seed(1221)

standard_claude10 = read_csv("AI_goldstandard_ten_v2.csv") %>% select(-...1)

#For limited sample...testing before full classification...%>% filter(docid %in% sample$docid)
excerpts_claude10 = standard_claude10 %>% select(docid, text) %>% arrange(docid) 
#Seed includes initializer row, so no need to add it in

# Prepare prompts with articles in same object
zero_labels <- c(z8_lab1, z8_lab2, z8_lab3, z8_lab4, z8_lab5, z8_lab6, z8_lab7, z8_lab8)
one_labels <- c(o8_lab1, o8_lab2, o8_lab3, o8_lab4, o8_lab5, o8_lab6, o8_lab7, o8_lab8)
few_labels <- c(f8_lab1, f8_lab2, f8_lab3, f8_lab4, f8_lab5, f8_lab6, f8_lab7, f8_lab8)

# Function to generate randomized prompts
generate_prompt <- function(labels, pre_text, post_text, text) {
  randomized_labels <- sample(labels)
  rand_text <- paste(randomized_labels, collapse = " ")
  prompt_text <- paste0(" <text> ", text, " </text>", pre_text, " ", rand_text, " ", post_text)
  return(prompt_text)
}

# Generate randomized prompts for each row
excerpts_claude10 <- excerpts_claude10 %>%
  mutate(
    prompt_zero = map_chr(text, ~generate_prompt(zero_labels, clpre_zero, clpost_zero, .x)),
    prompt_one = map_chr(text, ~generate_prompt(one_labels, clpre_one, clpost_one, .x)),
    prompt_few = map_chr(text, ~generate_prompt(few_labels, clpre_few, clpost_few, .x))
  )

```


```{r run everything, eval = F}
#Get feedback
responses_claude10_zero <- pbapply::pblapply(excerpts_claude10$prompt_zero,
                               fetch_response_claude,
                               cl = 1) #Clusters = 1
```


```{r run everything, eval = F}
#Get feedback
responses_claude10_one <- pbapply::pblapply(excerpts_claude10$prompt_one,
                               fetch_response_claude,
                               cl = 1) #Clusters = 1
```


```{r run everything, eval = F}
#Get feedback
responses_claude10_few <- pbapply::pblapply(excerpts_claude10$prompt_few,
                               fetch_response_claude,
                               cl = 1) #Clusters = 1
```


```{r examine and clean output}
#Extract results from JSON responses and clean them
results_claude10 <-
  lapply(
    responses_claude10_few,
    function(response) {
            r = gsub("json|\\n|```", "", response$content[[1]]$text)
           
            r = gsub("'", '"', r)
      
            try(as_tibble(fromJSON(r)))
    }
  )

cleaned_claude10 = as_tibble(t(bind_cols(results_claude10))) %>% 
  mutate(label = as.numeric(V1), 
         theme = rep(c("Theme1", "Theme2", "Theme3", "Theme4", "Theme5", "Theme6", "Theme7",
                       "Theme8"), times = 590)) #times = sample size

cleaned_claude10$docid = rep(excerpts_claude10$docid, each = 8)

#Pivot wider to match standard
clean_results_wide_claude10 = pivot_wider(cleaned_claude10, id_cols = "docid", 
                                          names_from = "theme", 
                            values_from = c("label"))

#Check the loss
#Reduce standard to size of test sample object and filter for hamming loss analysis
standard_ham_claude10 = standard_claude10 %>% 
  filter(docid %in% clean_results_wide_claude10$docid) %>% 
  rename(Theme1 = MAIN_HUM_HUMAN, Theme2 = MAIN_HUM_REFUGE, Theme3 = MAIN_THREAT_DIS, 
         Theme4 = MAIN_THREAT_ECON, 
         Theme5 = MAIN_THREAT_VIOL, Theme6 = MAIN_THREAT_INSTAB, Theme7 = MAIN_BEN_ECON, 
         Theme8 = MAIN_PI) %>% arrange(docid) %>% select(-c(text, docid))

clean_results_ham_claude10 = clean_results_wide_claude10 %>% arrange(docid) %>%
  select(Theme1:Theme8)

#Rename variables
names(clean_results_ham_claude10) = c("Theme1", "Theme2", "Theme3", "Theme4", "Theme5", "Theme6",
                                      "Theme7", "Theme8")

#Filter to sample rows...not going to change anything for final run because it's using the pop.
standard_test_claude10 = standard_claude10 %>% select(MAIN_HUM_HUMAN:MAIN_PI)
  #filter(docid %in% sample$docid) %>% 

names(standard_test_claude10) = c("Theme1", "Theme2", "Theme3", "Theme4", "Theme5", "Theme6",
                                  "Theme7", "Theme8")

#Run hamming loss function
hamming_loss(standard_test_claude10, clean_results_ham_claude10)

#save classifications for understanding of temporal changes
#write.csv(clean_results_wide_claude10, ".csv")
```

# Now, 4 label Claude

```{r load in data}
set.seed(1221)

standard_claude4 = read_csv("AI_goldstandard_four_v2.csv") %>% select(-...1)

#For limited sample...testing before full classification...filter(docid %in% sample$docid) %>%
excerpts_claude4 = standard_claude4 %>% select(docid, text) %>% 
   arrange(docid) #Seed includes initializer row, so no need to add it in

# Prepare prompts with articles in same object
zero_labels <- c(z4_lab1, z4_lab2, z4_lab3, z4_lab4)
one_labels <- c(o4_lab1, o4_lab2, o4_lab3, o4_lab4)
few_labels <- c(f4_lab1, f4_lab2, f4_lab3, f4_lab4)

# Function to generate randomized prompts
generate_prompt <- function(labels, pre_text, post_text, text) {
  randomized_labels <- sample(labels)
  rand_text <- paste(randomized_labels, collapse = " ")
  prompt_text <- paste0(" <text> ", text, " </text>", pre_text, " ", rand_text, " ", post_text)
  return(prompt_text)
}

# Generate randomized prompts for each row
excerpts_claude4 <- excerpts_claude4 %>%
  mutate(
    prompt_zero = map_chr(text, ~generate_prompt(zero_labels, clpre_zero, clpost_zero, .x)),
    prompt_one = map_chr(text, ~generate_prompt(one_labels, clpre_one, clpost_one, .x)),
    prompt_few = map_chr(text, ~generate_prompt(few_labels, clpre_few, clpost_few, .x))
  )
```


```{r run everything, eval = F}
#Get feedback
responses_claude4_zero <- pbapply::pblapply(excerpts_claude4$prompt_zero,
                               fetch_response_claude,
                               cl = 1) #Clusters = 1
```


```{r run everything, eval = F}
#Get feedback
responses_claude4_one <- pbapply::pblapply(excerpts_claude4$prompt_one,
                               fetch_response_claude,
                               cl = 1) #Clusters = 1
```


```{r run everything, eval = F}
#Get feedback
responses_claude4_few <- pbapply::pblapply(excerpts_claude4$prompt_few,
                               fetch_response_claude,
                               cl = 1) #Clusters = 1
```


```{r examine and clean output}
#Extract results from JSON responses and clean them
results_claude4 <-
  lapply(
    responses_claude4_few,
    function(response) {
            r = gsub("json|\\n|```", "", response$content[[1]]$text)
           
            r = gsub("'", '"', r)
      
            try(as_tibble(fromJSON(r)))
    }
  )

cleaned_claude4 = as_tibble(t(bind_cols(results_claude4))) %>% 
  #rename(label = V1, explanation = V2) %>% 
  mutate(label = as.numeric(V1), 
         theme = rep(c("Theme1", "Theme2", "Theme3", "Theme4"), times = 590)) #times = sample size

cleaned_claude4$docid = rep(excerpts_claude4$docid, each = 4)

#Pivot wider to match standard
clean_results_wide_claude4 = pivot_wider(cleaned_claude4, id_cols = "docid", 
                                         names_from = "theme", 
                            values_from = c("label")) #, "explanation"

#Check the loss
#Reduce standard to size of test sample object and filter for hamming loss analysis
standard_ham_claude4 = standard_claude4 %>% 
  #filter(docid %in% clean_results_wide_claude4$docid) %>% 
  rename(Theme1 = human, Theme2 = threat, Theme3 = benefit, 
         Theme4 = pi) %>% arrange(docid) %>% select(-c(text, docid))

clean_results_ham_claude4 = clean_results_wide_claude4 %>% arrange(docid) %>%
  select(Theme1:Theme4)

#Rename variables
names(clean_results_ham_claude4) = c("Theme1", "Theme2", "Theme3", "Theme4")

#Run hamming loss function
#Filter to sample rows
standard_test_claude4 = standard_claude4 %>% #filter(docid %in% sample$docid) %>%
  select(human:pi)

names(standard_test_claude4) = c("Theme1", "Theme2", "Theme3", "Theme4")

hamming_loss(standard_test_claude4, clean_results_ham_claude4)

#save classifications for understanding of temporal changes
#write.csv(clean_results_wide_claude4, ".csv") 
```

#Batching for ChatGPT 4o

```{r, message = F}

rm(list = ls())

library(tidyverse)
library(jsonlite)
library(httr)
library(pbapply)
library(tictoc)

# Function to write JSONL directly from a list of lists
write_jsonl <- function(data_list, file_name) {
  # Open a connection to the file
  con <- file(file_name, open = "wt")
  
  # Write each element of the list as a line in the JSONL file
  for (i in seq_along(data_list)) {
    # Convert the list to a JSON string and write it as a line
    writeLines(toJSON(data_list[[i]], auto_unbox = TRUE), con)
  }
  
  # Close the file connection
  close(con)
}

#Read in batch output and clean
read_and_parse_jsonl <- function(file_path) {
    jsonl_data = readLines(file_path)
    parsed_data = lapply(jsonl_data, jsonlite::fromJSON)
    
    results = lapply(
        parsed_data,
        function(x) {
            r = gsub("json|\\n|```", "", x$response$body$choices$message$content)
            r = gsub("'", '"', r)
            as_tibble(jsonlite::fromJSON(r))
        }
    )
    return(results)
}
```

#Colombia

```{r}
#Load in data and rename cols...note that I do not include this file because it contains the raw text data
p8 = read_csv("full_df_for_classification_ten.csv") %>% select(-...1)
```


```{r prep prompt}
#Create prompt
excerpts = p8 %>% select(docid, text) %>% arrange(docid)

#Prepare prompts with articles in same object
prompt_text = 
  readLines("chatgpt_prompt_ten_fewshot v7.txt", warn = FALSE, 
            encoding = "UTF-8") |>
  paste(collapse = " ") |>
  gsub("  ", " ", x = _) |>
  gsub('[\"]', "'", x = _) 

excerpts$prompt = paste(prompt_text, "<text>", excerpts$text, "</text>")

#Make into chat completion format
# Initialize an empty list to hold all the request objects
requests <- list()

# Loop through each prompt and completion
for (i in seq_along(excerpts$prompt)) {
  
  # Define each part of the conversation
  system_part <- list(role = "system", content = "You are a research coder for newspaper articles")
  user_part <- list(role = "user", content = excerpts$prompt[i])
  #assistant_part <- list(role = "assistant", content = [i])
  
  # Combine parts into a single message set
  all_messages <- list(system_part, user_part)
  
  # Create the request object
  request <- list(
    custom_id = paste("request-", i, sep = ""),
    method = "POST",
    url = "/v1/chat/completions",
    body = list(
      model = "gpt-4o",
      messages = all_messages,
      max_tokens = 1000,
      response_format = list(type = "json_object")
    )
  )
  
  # Append the complete request object to the main list
  requests[[i]] <- request
}

```


```{r write to jsonl}
#Full...split because of size constraints
#write_jsonl(requests[1:4500], "col_prompts_part1_v2.jsonl")
#write_jsonl(requests[4501:9009], "col_prompts_part2_v2.jsonl")
```

#v2 has changes based on close reading of original (more specific info, more fine-tuned examples, etc.)

```{r parse batch output}
#Read in output files
results_p1 = read_and_parse_jsonl("batch_part1_output_v2.jsonl")
results_p2 = read_and_parse_jsonl("batch_part2_output_v2.jsonl")

clean_results = bind_rows(results_p1, results_p2, .id = "id") %>% as_tibble()

# Find indices of lists that do not have a length of 8
#which(sapply(results_p2, function(x) length(x) != 8)) #All clear

#Identify duplicate rows
#duplicated_rows <- clean_results %>%
#  filter(duplicated(.) | duplicated(., fromLast = TRUE))

#Remove duplicate rows from merge
clean_results_unique = clean_results %>% distinct(id, .keep_all = T)

#Now add in docids
clean_results_unique$docid = rep(excerpts$docid, each = 1)

#Merge in meta data, remove unique id column, remove initializer row
final_df = left_join(clean_results_unique, p8, by = "docid") %>% select(-id) %>% filter(docid != 0)
#anti_join(clean_results_unique, p8, by = "docid") #No issues merging

#Save
#write.csv(final_df, ".csv")
```


```{r}

```

